# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

import pymysql
# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import re
import emoji
from MySpider import mydb


def com_db_process(item, spider):
    """
    将item写入数据库公共处理流程
    :param item:
    :param spider:
    :return:
    """
    values = ['%s' % str(item.get(i, '')) for i in spider.table_fields]

    try:
        mydb.cursor.execute(spider.insert_sql, values)
    except Exception as e:
        print('第' + str(item['page']) + '页，第' + str(item['page_poz']) + '个 异常')
        print(spider.insert_sql)
        print(tuple(values))
        print(e)
        # 解决emoji表情存储失败问题
        if 'Incorrect string value' in e.__str__():
            # 哪个字段出错解决哪个
            re_correct_key = re.search(r'(?<=for column \').*(?=\' at row)', e.__str__()).group(0)
            # 重新filter一下
            values[spider.table_fields.index(re_correct_key)] = emoji.replace_emoji(
                values[spider.table_fields.index(re_correct_key)])
            mydb.cursor.execute(spider.insert_sql, values)
            print('异常已处理')
            print('--------------------------')
        else:
            print('异常未处理')
            print('--------------------------')
    finally:
        # 要提交，不提交无法保存到数据库
        mydb.connect.commit()


class MyPipeline:

    def __init__(self):
        self.keys = None
        self.values_params = None

    def open_spider(self, spider):
        pass

    def close_spider(self, spider):
        pass

    def process_item(self, item, spider):
        # 先运行公共处理流程————将数据入库
        com_db_process(item, spider)
        return item


